package com.g2room.driving.grap.article.kc;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.g2room.driving.grap.article.to.ArticlesTo;
import com.g2room.driving.model.po.Article;
import com.g2room.driving.util.DbUtil;
import com.g2room.driving.util.IpUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSpan;
import com.gargoylesoftware.htmlunit.html.HtmlTable;
import com.gargoylesoftware.htmlunit.html.HtmlTableCell;
import com.gargoylesoftware.htmlunit.html.HtmlTableRow;

public class GrabKcjqDataByProxy {
	static Connection connection = null;
	static PreparedStatement  ps = null;
	static Map<String, String> errorUrls = new HashMap<String, String>();
	private static final String homePage = "http://www.kcjq.com";
	
	private static Connection getConn() throws ClassNotFoundException, SQLException {
		return connection = DbUtil.getConn();
	}
	static Map<String, String> errMap = new HashMap<String, String>();
	public static void main(String[] args) throws Exception{
		GrabKcjqDataByProxy gkd = new GrabKcjqDataByProxy();
		gkd.getDetails();
		
////		gkd.getNav(homePage);
//		// 最后输出失败的链接到数据库
////		gkd.getDetail("http://www.kcjq.com/news/03/2009/1214/0912141013493CDA28B3C0GB4036EDBD.html");
//		for(String url : errMap.keySet()) {
//			ArticlesTo at = new ArticlesTo();
//			at.setTitle(errMap.get(url));
//			at.setSourceUrl(url);
//			gkd.getDetail(url, at);
//		}
////		gkd.getNav("http://www.kcjq.com/news/01/20151117/19584.html");
//		insertError2Db();
    }
	
	private WebClient getWebClient() {
		 //创建一个webclient
		WebClient wc = new WebClient();
		String ip = IpUtil.getIp();
		wc.addRequestHeader("X-Real-IP", ip);
		wc.addRequestHeader("x-forwarded-for", ip);
		wc.addRequestHeader("WL-Proxy-Client-IP", ip);
		////设置请求报文头里的User-Agent字段
        wc.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2");
        //wc.addRequestHeader和request.setAdditionalHeader功能应该是一样的。选择一个即可。
        //其他报文头字段可以根据需要添加
        wc.getCookieManager().setCookiesEnabled(false);//开启cookie管理
        wc.getOptions().setJavaScriptEnabled(false);//开启js解析。对于变态网页，这个是必须的
        wc.getOptions().setCssEnabled(false);//开启css解析。对于变态网页，这个是必须的。
        wc.getOptions().setThrowExceptionOnFailingStatusCode(false);
        wc.getOptions().setThrowExceptionOnScriptError(false);
        wc.getOptions().setTimeout(10000);
        
        //htmlunit 对css和javascript的支持不好，所以请关闭之
        wc.getOptions().setJavaScriptEnabled(false);
        wc.getOptions().setCssEnabled(false);
        return wc;
	}
	
	/**
	 * 获取头部导航菜单
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getNav(String url) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
        WebClient wc = getWebClient();
		//获取页面
        HtmlPage page = null;
        try {
        	page = wc.getPage(url);
        } catch (Exception e) {
        	errorUrls.put(url, "导航页");
        	e.printStackTrace();
        	return;
        }
        //查找并获取特定input
        List<?> list = page.getByXPath("//a[@class='menu']");
        Map<String, String> pages = new HashMap<String, String>();
        for(Object hs : list) {
        	HtmlAnchor htmlAnchor = (HtmlAnchor) hs;
        	pages.put(htmlAnchor.getAttribute("href"), htmlAnchor.asText());
        }
        if (null != wc) wc.close();
        
        for(String key : pages.keySet()) {
	        try {
	        	Thread.sleep(IpUtil.genRandomNum() * 500);
	        } catch (InterruptedException e) {
	        	// TODO Auto-generated catch block
	        	e.printStackTrace();
	        }
	        getPage(homePage + key, pages.get(key));
        }
	}
	
	/**
	 * 获取分页数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getPage(String url, String typeName) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get page data!" + typeName);
		WebClient wc = getWebClient();
		//获取页面
		HtmlPage page = null;
        try {
        	page = wc.getPage(url);
        } catch (Exception e) {
        	errorUrls.put(url, typeName);
        	e.printStackTrace();
        	return;
        }
		getList(url, typeName);
		// 获取分页数据
		List<?> list = page.getByXPath("//a[@style='color: #CC0066;']");
		List<String> hrefs = new ArrayList<String>();
		for(Object hs : list) {
			HtmlAnchor htmlAnchor = (HtmlAnchor) hs;
			hrefs.add(htmlAnchor.getAttribute("href"));
		}
        if (null != wc) wc.close();
        
        for(String href : hrefs) {
        	try {
				Thread.sleep(IpUtil.genRandomNum() * 500);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
			getList(homePage + href, typeName);
        }
	}
	
	/**
	 * 获取列表数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getList(String url, String typeName) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get list data!url==  " + url + "   ,typeName=" + typeName);
		WebClient wc = getWebClient();
		//获取页面
		HtmlPage page = null;
        try {
        	page = wc.getPage(url);
        } catch (Exception e) {
        	errorUrls.put(url, typeName);
        	e.printStackTrace();
        	return;
        }
		// 获取分页数据
		List<?> list = page.getByXPath("//table[@width='94%']");
		Map<String, ArticlesTo> details = new HashMap<String, ArticlesTo>();
		for(Object hs : list) {
			HtmlTable table = (HtmlTable) hs;
			
			for (final HtmlTableRow row : table.getBodies().get(0).getRows()) {
				ArticlesTo articles = new ArticlesTo();
//				articles.setTypeId(typeId);
				articles.setTypeName(typeName);
				String href = null;
				for (final HtmlTableCell cell : row.getCells()) {
					String tempHref = getTagAText("a", "href", cell.asXml());
					if(StringUtils.isEmpty(tempHref)) {
						articles.setPublishTime(cell.asText());
						continue;
					}
					href = tempHref;
					articles.setTitle(getTagAText("a", "title", cell.asXml()));
				}
				details.put(href, articles);
			}
		}
        if (null != wc) wc.close();
        
        for(String key : details.keySet()) {
        	try {
        		Thread.sleep(IpUtil.genRandomNum() * 500);
        	} catch (InterruptedException e) {
        		e.printStackTrace();
        	}
        	getDetail(homePage + key, details.get(key));
        }
	}
	
	private String getTagAText(String element, String attr, String str) {
		String patternStr = "<" + element + "[^<>]*?\\s" + attr + "=['\"]?(.*?)['\"]?\\s.*?>";
//		patternStr = "<a[^>]*>([^<]*)</a>";
	      return getTagAText(patternStr, str);
	}
	
	private String getTagAText(String patternStr, String str) {
		Pattern p = Pattern.compile(patternStr);
		Matcher m = p.matcher(str);
		String rs = null;
		while (m.find()) {
			rs = m.group(1);
		}
		return rs;
	}
	
	public void getDetail(String url, ArticlesTo articles) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get detail data!  " + url + ",  title=" + articles.getTitle());
		WebClient wc = getWebClient();
		HtmlPage page = null;
        try {
        	page = wc.getPage(url);
        } catch (Exception e) {
        	errorUrls.put(url, articles.getTitle());
        	e.printStackTrace();
        	return;
        }
        articles.setSourceUrl(url);
		// 获取分页数据
		List<?> list = page.getByXPath("//span[@class='wenzhan']");
		for(Object hs : list) {
			HtmlSpan span = (HtmlSpan) hs;
			articles.setContent(span.asXml().replaceAll("　", " "));
			insert2Db(articles);
		}
        if (null != wc) wc.close();
//		try {
//			if(null != ps) ps.close();
//			if(null != connection) connection.close();
//		} catch (SQLException e) {
//			e.printStackTrace();
//		}
	}
	int sqlCount = 0;
	public void insert2Db(ArticlesTo articles) {
		try {
			if(sqlCount == 0) ps = getConn().prepareStatement("insert into xcjq(title,content,type_name,publish_time,creat_time,url) values(?,?,?,?,?,?,?)");
			ps.setString(1, articles.getTitle());
			ps.setString(2, articles.getContent());
			ps.setString(3, articles.getTypeName());
			ps.setString(4, articles.getPublishTime());
			ps.setTimestamp(5, new Timestamp(System.currentTimeMillis()));
			ps.setString(6, articles.getSourceUrl());
			ps.execute();
//			ps.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		/**
		try {
			if(sqlCount == 0) ps = getConn().prepareStatement("insert into xcjq(title,content,type_id,type_name,publish_time,creat_time) values(?,?,?,?,?,?)");
			ps.setString(1, articles.getTitle());
			ps.setString(2, articles.getContent());
			ps.setInt(3, articles.getTypeId());
			ps.setString(4, articles.getTypeName());
			ps.setString(5, articles.getPublishTime());
			ps.setTimestamp(6, new Timestamp(System.currentTimeMillis()));
			ps.addBatch();
			sqlCount++;
			if(sqlCount == 50) {
				System.out.println("----------------------ready insert to DB! title=" + articles.getTitle());
				ps.executeBatch();
				ps.close();
				sqlCount = 0;
			}
		} catch(Exception e) {
			e.printStackTrace();
		}
		*/
	}
	
	public static void insertError2Db() {
		try {
			PreparedStatement  ps = getConn().prepareStatement("insert into xcjq_error(title,url) values(?,?)");
			for(String url : errorUrls.keySet()) {
				ps.setString(1, errorUrls.get(url));
				ps.setString(2, url);
				ps.addBatch();
			}
			ps.executeBatch();
			if(null != ps) ps.close();
			if(null != connection) connection.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	
	
	// ----------------------------------以下是单独抓取部分数据
	public void getDetails() throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		List<Article> articles = getArticles();
		for(Article arti : articles) {
			WebClient wc = getWebClient();
			HtmlPage page = null;
	        try {
	        	page = wc.getPage(arti.getSourceUrl());
	        } catch (Exception e) {
	        	System.out.println(arti.getId() + "============" + arti.getSourceUrl());
	        	e.printStackTrace();
	        	return;
	        }
	     // 获取分页数据
			List<?> list = page.getByXPath("//span[@class='wenzhan']");
			for(Object hs : list) {
				HtmlSpan span = (HtmlSpan) hs;
				arti.setContent(span.asXml().replaceAll("　", " "));
				updateArtiDb(arti);
			}
	        if (null != wc) wc.close();
		}
		try {
			if(null != connection) connection.close();
		} catch (SQLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	
	public static void updateArtiDb(Article arti) {
		try {
			PreparedStatement  ps = getConn().prepareStatement("update d_article_grap set content=?,content_origin=? where id=?");
			ps.setString(1, handleKcjqTag(arti.getContent()));
			ps.setString(2, arti.getContent());
			ps.setLong(3, arti.getId());
			ps.execute();
			if(null != ps) ps.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	public static List<Article> getArticles() {
		List<Article> articles = new ArrayList<Article>();
		try {
			Connection connection = DbUtil.getConn();
			Statement statement = connection.createStatement();
			// 查询最新10条记录
			String sql =  "select id,source_url from d_article_grap where id in (226,236,237,241,242,245,246,251,259,269,289,296,297,303,304,317,463,467,471,479,483,488,489,491,492,493,503,506,512,519,528,572,576,578,586,590,592,593,594,597,598,608,619,635,639,758,761,762,767,769,779,780,781,782,785,798,799,800,811,1299,1304,1305,1306,1308,1309,1310,1321,1324,1325,1326,1337,1347,1377,1409,1418,1421,1429,1437,1440,2229,2235,2236,2260,2401,2402,2407,2417,2419,2424,2435,2437,2439,2442,2443,2453,2454,2456,2458,2462,2473,2476,2483,2490,2491,2492,2494,2495,2496,2497,2499,2500,2505,2510,2515,2523,2538,2539,2543,2546,2549,2551,2562,2564,2565,2567,2585,2595,2604,2605,2617,2627,2645,2686,2694,2702,2709,2711,2713,2714,2721,2732,2733,2744,2749,2756,2779,2786,2790,2799,2803,2828,2832,2833,2838,2839,2847,2855,2861,2875,2878,2884,2887,2891,2905,2907,2917,2925,2926,2927,2934,2941,2942,2954,2970,2983,2985,3055,3056,3059,3062,3063,3066,3068,3075,3076,3077,3080,3085,3093,3096,3098,3123,3124,3125,3126,3127,3130,3136,3161,3172,3177,3183,3189)";
			ResultSet rs = statement.executeQuery(sql);
			while (rs.next()) {
				Article article = new Article();
				article.setId(rs.getLong("id"));
				article.setSourceUrl(rs.getString("source_url"));
				articles.add(article);
			}
			rs.close();
			statement.close();
//			connection.close();
		} catch (Exception e) {
			e.printStackTrace();
		}
		return articles;
	}
	
	public static String handleKcjqTag(String content) {
		try{
            //实例化一个本地html文件的Parser
			String html = content;
			Document doc = Jsoup.parse(html);
			Elements els = doc.getElementsByAttributeValue("class", "pagecontent");
			if(null == els || els.size() == 0) return content;
			// 获取第一层class="acticle"
			els.get(0).remove();
			Elements divEls = doc.getElementsByTag("div");
			for (Element el : divEls) {
				// 获取div下第一层元素
				String text = el.text();
				if(text.trim().length() == 0) el.remove();
			}
			
			content = handleSpace(doc.body().html());
        }catch(Exception e) {
        	System.out.println("content========" + content);
        	e.printStackTrace();
        }
		return content;
	}
	
	public static String handleSpace(String str) {
		return str.replaceAll("&nbsp;", " ");
	}
}
